/*
 * To change this template, choose Tools | Templates
 * and open the template in the editor.
 */
package analisador;

import java.io.IOException;
import java.io.Reader;
import java.io.StringReader;
import java.io.StringWriter;
import java.io.Writer;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.TokenStream;
import org.apache.lucene.analysis.tokenattributes.TermAttribute;
import org.apache.lucene.demo.html.HTMLParser;
import org.apache.lucene.util.Version;

/**
 *
 * @author andre
 */
public class Teste {

    public static void main(String[] args) throws Exception {

        String string = " Hi Pete, The block stuff in fuse is appallingly documented. I have hunted the Web for info on this all afternoon, to understand it further. To be honest, the only thing I have found useful is reading the source of ntfs-3g.c at http://ntfs-3g.cvs.sourceforge.net/ntfs-3g/ntfs-3g/src/ntfs-3g.c?revision=1.106&view=markup I test I did do a few days ago was to comparing reading an NFS mounted file directly vs, the same file read via NFS via a FUSE fs - http://mattwork.potsdam.edu/projects/wiki/index.php/Rofs#rofs_code_.28C.29 (ROFS, the Read-Only Filesystem). Speed results were fairly comparable between NFS & NFS+ROFS, so it suggests that FUSE doesnt add too much overhead to IO. Hence then we can only suspect that the problem is in either (a) JNI interface, or (b) the size of the reads we're performing. A simple C tool can be generated to exclude (a). I dont have any objections to pretty large buffer sizes for fuse_dfs.c - HDFS is designed for large files, and streaming read access. Btw, you mentioned you are re-exporting the mounted FS as NFS - have you had any issues vs the issues described in fuses' README.NFS? Regards Craig http://ntfs-3g.cvs.sourceforge.net/ntfs-3g/ntfs-3g/src/ntfs-3g.c?revision=1.106&view=markup";
        Analyzer an = new CustomAnalyzer(Version.LUCENE_35);

        HTMLParser htmlParser = new HTMLParser(new StringReader(string));

//        String str = convertReaderToString(htmlParser.getReader());
        StringBuilder str = new StringBuilder(convertReaderToString(htmlParser.getReader()));
        
        
        
        int start;
        do{
            start = str.indexOf("http://");
            if (start != -1) {
                int end = str.indexOf(" ", start);
                if (end == -1){
                    end = str.length();
                }
                String s = str.substring(start, end);
                int p = str.indexOf(s);
                System.out.println(s);
                str = str.replace(start,end," ");
            }
        }while(start != -1);

//        TokenStream stream = an.tokenStream("comentario", new StringReader(str));
        TokenStream stream = an.tokenStream("comentario", new StringReader(str.toString()));

        TermAttribute term = stream.addAttribute(TermAttribute.class);
        while (stream.incrementToken()) {
            System.out.println(term.term());
        }
    }

    public static String convertReaderToString(Reader reader)
            throws IOException {

        if (reader == null) {
            return "";
        }
        Writer writer = new StringWriter();

        char[] buffer = new char[1024];

        int n;

        while ((n = reader.read(buffer)) != -1) {
            writer.write(buffer, 0, n);
        }

        return writer.toString();
    }
}
